{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c25e5705-7078-4d4a-9fa2-8aaa528ffced",
   "metadata": {},
   "source": [
    "# Week 7 Day 2\n",
    "\n",
    "# \"THE PRICE IS RIGHT\" Capstone Project\n",
    "\n",
    "This week - fine tune an open-source model!\n",
    "\n",
    "A model that can estimate how much something costs, from its description.\n",
    "\n",
    "# Order of play\n",
    "\n",
    "DAY 1: QLoRA  \n",
    "DAY 2: Prompt data and Base Model  \n",
    "DAY 3: Train Part 1  \n",
    "DAY 4: Train Part 2  \n",
    "DAY 5: Eval \n",
    "\n",
    "## First we need to upload the final dataset\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "567a27ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "from huggingface_hub import login\n",
    "from pricer.items import Item\n",
    "from tqdm.notebook import tqdm\n",
    "from transformers import AutoTokenizer\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7e1986c",
   "metadata": {},
   "outputs": [],
   "source": [
    "LITE_MODE = False\n",
    "\n",
    "load_dotenv(override=True)\n",
    "hf_token = os.environ['HF_TOKEN']\n",
    "login(hf_token, add_to_git_credential=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d76924e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "username = \"ed-donner\"\n",
    "dataset = f\"{username}/items_lite\" if LITE_MODE else f\"{username}/items_full\"\n",
    "\n",
    "train, val, test = Item.from_hub(dataset)\n",
    "items = train + val + test\n",
    "\n",
    "print(f\"Loaded {len(train):,} training items, {len(val):,} validation items, {len(test):,} test items\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7819cad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_MODEL = \"meta-llama/Llama-3.2-3B\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "707af732",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_counts = [item.count_tokens(tokenizer) for item in tqdm(items)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a889cfac",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 6))\n",
    "plt.title(f\"Tokens in Summary: Avg {sum(token_counts)/len(token_counts):,.1f} and highest {max(token_counts):,}\\n\")\n",
    "plt.xlabel('Number of tokens in summary')\n",
    "plt.ylabel('Count')\n",
    "plt.hist(token_counts, rwidth=0.7, color=\"skyblue\", bins=range(0, 200, 10))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9db5ddf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "CUTOFF = 110\n",
    "cut = len([count for count in token_counts if count > CUTOFF])\n",
    "print(f\"With this CUTOFF, we will truncate {cut:,} items which is {cut/len(items):.1%}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be726e7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train[0].summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48e9b998",
   "metadata": {},
   "outputs": [],
   "source": [
    "for item in tqdm(train+val):\n",
    "    item.make_prompts(tokenizer, CUTOFF, True)\n",
    "for item in tqdm(test):\n",
    "    item.make_prompts(tokenizer, CUTOFF, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "055a24c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"PROMPT:\")\n",
    "print(test[0].prompt)\n",
    "print(\"COMPLETION:\")\n",
    "print(test[0].completion)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91487564",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_token_counts = [item.count_prompt_tokens(tokenizer) for item in tqdm(items)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f71d1754",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(15, 6))\n",
    "plt.title(f\"Tokens: Avg {sum(prompt_token_counts)/len(prompt_token_counts):,.1f} and highest {max(prompt_token_counts):,}\\n\")\n",
    "plt.xlabel('Number of tokens in prompt and the completion')\n",
    "plt.ylabel('Count')\n",
    "plt.hist(prompt_token_counts, rwidth=0.7, color=\"gold\", bins=range(0, 200, 10))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6a4b745",
   "metadata": {},
   "outputs": [],
   "source": [
    "username = \"ed-donner\"\n",
    "dataset = f\"{username}/items_prompts_lite\" if LITE_MODE else f\"{username}/items_prompts_full\"\n",
    "\n",
    "Item.push_prompts_to_hub(dataset, train, val, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff661da1",
   "metadata": {},
   "source": [
    "Here are the datasets in HuggingFace:\n",
    "\n",
    "https://huggingface.co/datasets/ed-donner/items_prompts_lite\n",
    "\n",
    "https://huggingface.co/datasets/ed-donner/items_prompts_full\n",
    "\n",
    "Please see this notebook in Google Colab:\n",
    "\n",
    "https://colab.research.google.com/drive/1wO3lNMrMfprlJZF4X9fSsQ8tYC3SRZbh?usp=sharing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3ea2c7c",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
